{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cleanlab import noise_generation\n",
    "import torchvision\n",
    "from torchvision import transforms\n",
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import json\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Prepare CIFAR-10 dataset for PyTorch dataloader\n",
    "# import os\n",
    "# for kind in ['test/', 'train/']:\n",
    "#     data_path = '/datasets/datasets/cifar10/cifar10/'\n",
    "#     for file in os.listdir(data_path + kind):\n",
    "#         class_name = file.split('_')[-1].split('.')[0]\n",
    "#         os.system('mv {} {}'.format(data_path+kind+file, data_path+kind+class_name+\"/\"+file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create json with train labels\n",
    "for cifar_dataset in [\"cifar10\", \"cifar100\"]:\n",
    "    data_path = '/datasets/datasets/{}/{}/'.format(cifar_dataset, cifar_dataset)\n",
    "    train_dataset = torchvision.datasets.ImageFolder(data_path + 'train/')\n",
    "    d = dict(train_dataset.imgs)\n",
    "    # Store the dictionary        \n",
    "    with open(data_path + \"train_filename2label.json\", 'w') as wf:\n",
    "        wf.write(json.dumps(d, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create json with test labels\n",
    "for cifar_dataset in [\"cifar10\", \"cifar100\"]:\n",
    "    data_path = '/datasets/datasets/{}/{}/'.format(cifar_dataset, cifar_dataset)\n",
    "    test_dataset = torchvision.datasets.ImageFolder(data_path + 'test/')\n",
    "    d = dict(test_dataset.imgs)\n",
    "    # Store the dictionary        \n",
    "    with open(data_path + \"test_filename2label.json\", 'w') as wf:\n",
    "        wf.write(json.dumps(d, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cifar10 [5000 5000 5000 5000 5000 5000 5000 5000 5000 5000]\n",
      "noise_amount 0.0 | frac_zero_noise_rates 0.0\n",
      "valid: True\n",
      "Accuracy of s and y: 1.0\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.0__noise_amount__0.0\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.0__noise_amount__0.0\n",
      "noise_amount 0.2 | frac_zero_noise_rates 0.0\n",
      "valid: True\n",
      "Accuracy of s and y: 0.80086\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.0__noise_amount__0.2\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.0__noise_amount__0.2\n",
      "noise_amount 0.4 | frac_zero_noise_rates 0.0\n",
      "valid: True\n",
      "Accuracy of s and y: 0.60092\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.0__noise_amount__0.4\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.0__noise_amount__0.4\n",
      "noise_amount 0.6 | frac_zero_noise_rates 0.0\n",
      "valid: False\n",
      "Accuracy of s and y: 0.30088\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.0__noise_amount__0.6\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.0__noise_amount__0.6\n",
      "noise_amount 0.8 | frac_zero_noise_rates 0.0\n",
      "valid: False\n",
      "Accuracy of s and y: 0.10094\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.0__noise_amount__0.8\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.0__noise_amount__0.8\n",
      "noise_amount 0.0 | frac_zero_noise_rates 0.2\n",
      "valid: True\n",
      "Accuracy of s and y: 1.0\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.2__noise_amount__0.0\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.2__noise_amount__0.0\n",
      "noise_amount 0.2 | frac_zero_noise_rates 0.2\n",
      "valid: True\n",
      "Accuracy of s and y: 0.80068\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.2__noise_amount__0.2\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.2__noise_amount__0.2\n",
      "noise_amount 0.4 | frac_zero_noise_rates 0.2\n",
      "valid: False\n",
      "Accuracy of s and y: 0.60066\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.2__noise_amount__0.4\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.2__noise_amount__0.4\n",
      "noise_amount 0.6 | frac_zero_noise_rates 0.2\n",
      "valid: False\n",
      "Accuracy of s and y: 0.30082\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.2__noise_amount__0.6\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.2__noise_amount__0.6\n",
      "noise_amount 0.8 | frac_zero_noise_rates 0.2\n",
      "valid: False\n",
      "Accuracy of s and y: 0.10074\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.2__noise_amount__0.8\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.2__noise_amount__0.8\n",
      "noise_amount 0.0 | frac_zero_noise_rates 0.4\n",
      "valid: True\n",
      "Accuracy of s and y: 1.0\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.4__noise_amount__0.0\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.4__noise_amount__0.0\n",
      "noise_amount 0.2 | frac_zero_noise_rates 0.4\n",
      "valid: True\n",
      "Accuracy of s and y: 0.80054\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.4__noise_amount__0.2\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.4__noise_amount__0.2\n",
      "noise_amount 0.4 | frac_zero_noise_rates 0.4\n",
      "valid: True\n",
      "Accuracy of s and y: 0.60052\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.4__noise_amount__0.4\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.4__noise_amount__0.4\n",
      "noise_amount 0.6 | frac_zero_noise_rates 0.4\n",
      "valid: False\n",
      "Accuracy of s and y: 0.30054\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.4__noise_amount__0.6\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.4__noise_amount__0.6\n",
      "noise_amount 0.8 | frac_zero_noise_rates 0.4\n",
      "valid: False\n",
      "Accuracy of s and y: 0.10052\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.4__noise_amount__0.8\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.4__noise_amount__0.8\n",
      "noise_amount 0.0 | frac_zero_noise_rates 0.6\n",
      "valid: True\n",
      "Accuracy of s and y: 1.0\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.6__noise_amount__0.0\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.6__noise_amount__0.0\n",
      "noise_amount 0.2 | frac_zero_noise_rates 0.6\n",
      "valid: True\n",
      "Accuracy of s and y: 0.80034\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.6__noise_amount__0.2\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.6__noise_amount__0.2\n",
      "noise_amount 0.4 | frac_zero_noise_rates 0.6\n",
      "valid: False\n",
      "Accuracy of s and y: 0.60038\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.6__noise_amount__0.4\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.6__noise_amount__0.4\n",
      "noise_amount 0.6 | frac_zero_noise_rates 0.6\n",
      "valid: False\n",
      "Accuracy of s and y: 0.30036\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.6__noise_amount__0.6\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.6__noise_amount__0.6\n",
      "noise_amount 0.8 | frac_zero_noise_rates 0.6\n",
      "valid: False\n",
      "Accuracy of s and y: 0.10032\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noisy_labels__frac_zero_noise_rates__0.6__noise_amount__0.8\n",
      "/datasets/datasets/cifar10/cifar10/noisy_labels/cifar10_noise_matrix__frac_zero_noise_rates__0.6__noise_amount__0.8\n",
      "cifar100 [500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500\n",
      " 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500\n",
      " 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500\n",
      " 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500\n",
      " 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500\n",
      " 500 500 500 500 500 500 500 500 500 500]\n",
      "noise_amount 0.0 | frac_zero_noise_rates 0.0\n",
      "valid: True\n",
      "Accuracy of s and y: 1.0\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.0__noise_amount__0.0\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.0__noise_amount__0.0\n",
      "noise_amount 0.2 | frac_zero_noise_rates 0.0\n",
      "valid: True\n",
      "Accuracy of s and y: 0.87426\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.0__noise_amount__0.2\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.0__noise_amount__0.2\n",
      "noise_amount 0.4 | frac_zero_noise_rates 0.0\n",
      "valid: True\n",
      "Accuracy of s and y: 0.68242\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.0__noise_amount__0.4\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.0__noise_amount__0.4\n",
      "noise_amount 0.6 | frac_zero_noise_rates 0.0\n",
      "valid: False\n",
      "Accuracy of s and y: 0.48086\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.0__noise_amount__0.6\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.0__noise_amount__0.6\n",
      "noise_amount 0.8 | frac_zero_noise_rates 0.0\n",
      "valid: False\n",
      "Accuracy of s and y: 0.2846\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.0__noise_amount__0.8\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.0__noise_amount__0.8\n",
      "noise_amount 0.0 | frac_zero_noise_rates 0.2\n",
      "valid: True\n",
      "Accuracy of s and y: 1.0\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.2__noise_amount__0.0\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.2__noise_amount__0.0\n",
      "noise_amount 0.2 | frac_zero_noise_rates 0.2\n",
      "valid: True\n",
      "Accuracy of s and y: 0.86228\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.2__noise_amount__0.2\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.2__noise_amount__0.2\n",
      "noise_amount 0.4 | frac_zero_noise_rates 0.2\n",
      "valid: True\n",
      "Accuracy of s and y: 0.66736\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.2__noise_amount__0.4\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.2__noise_amount__0.4\n",
      "noise_amount 0.6 | frac_zero_noise_rates 0.2\n",
      "valid: False\n",
      "Accuracy of s and y: 0.4632\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.2__noise_amount__0.6\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.2__noise_amount__0.6\n",
      "noise_amount 0.8 | frac_zero_noise_rates 0.2\n",
      "valid: False\n",
      "Accuracy of s and y: 0.26554\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.2__noise_amount__0.8\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.2__noise_amount__0.8\n",
      "noise_amount 0.0 | frac_zero_noise_rates 0.4\n",
      "valid: True\n",
      "Accuracy of s and y: 1.0\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.4__noise_amount__0.0\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.4__noise_amount__0.0\n",
      "noise_amount 0.2 | frac_zero_noise_rates 0.4\n",
      "valid: True\n",
      "Accuracy of s and y: 0.84842\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.4__noise_amount__0.2\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.4__noise_amount__0.2\n",
      "noise_amount 0.4 | frac_zero_noise_rates 0.4\n",
      "valid: False\n",
      "Accuracy of s and y: 0.65248\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.4__noise_amount__0.4\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.4__noise_amount__0.4\n",
      "noise_amount 0.6 | frac_zero_noise_rates 0.4\n",
      "valid: False\n",
      "Accuracy of s and y: 0.44496\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.4__noise_amount__0.6\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.4__noise_amount__0.6\n",
      "noise_amount 0.8 | frac_zero_noise_rates 0.4\n",
      "valid: False\n",
      "Accuracy of s and y: 0.24684\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.4__noise_amount__0.8\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.4__noise_amount__0.8\n",
      "noise_amount 0.0 | frac_zero_noise_rates 0.6\n",
      "valid: True\n",
      "Accuracy of s and y: 1.0\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.6__noise_amount__0.0\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.6__noise_amount__0.0\n",
      "noise_amount 0.2 | frac_zero_noise_rates 0.6\n",
      "valid: True\n",
      "Accuracy of s and y: 0.83308\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.6__noise_amount__0.2\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.6__noise_amount__0.2\n",
      "noise_amount 0.4 | frac_zero_noise_rates 0.6\n",
      "valid: False\n",
      "Accuracy of s and y: 0.6359\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.6__noise_amount__0.4\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.6__noise_amount__0.4\n",
      "noise_amount 0.6 | frac_zero_noise_rates 0.6\n",
      "valid: False\n",
      "Accuracy of s and y: 0.42778\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.6__noise_amount__0.6\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.6__noise_amount__0.6\n",
      "noise_amount 0.8 | frac_zero_noise_rates 0.6\n",
      "valid: False\n",
      "Accuracy of s and y: 0.228\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noisy_labels__frac_zero_noise_rates__0.6__noise_amount__0.8\n",
      "/datasets/datasets/cifar100/cifar100/noisy_labels/cifar100_noise_matrix__frac_zero_noise_rates__0.6__noise_amount__0.8\n"
     ]
    }
   ],
   "source": [
    "# Create noisy labels for both CIFAR-10 and CIFAR-100\n",
    "for cifar_dataset in [\"cifar10\", \"cifar100\"]:\n",
    "    data_path = '/datasets/datasets/{}/{}/'.format(cifar_dataset, cifar_dataset)\n",
    "    train_dataset = torchvision.datasets.ImageFolder(\n",
    "        root=data_path + 'train/',\n",
    "        transform=transforms.Compose([\n",
    "            transforms.RandomCrop(32, padding=4),\n",
    "            transforms.RandomHorizontalFlip(),\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "        ]),\n",
    "    )\n",
    "    y = train_dataset.targets\n",
    "    K = int(cifar_dataset[5:])\n",
    "    print(cifar_dataset, np.bincount(y))\n",
    "    for frac_zero_noise_rates in np.arange(0, 0.8, 0.2):\n",
    "        for noise_amount in np.arange(0, 1, 0.2):\n",
    "            print('noise_amount', round(noise_amount, 1), '| frac_zero_noise_rates', round(frac_zero_noise_rates, 1))\n",
    "\n",
    "            # Generate class-conditional noise        \n",
    "            nm = noise_generation.generate_noise_matrix_from_trace(\n",
    "                K=K,\n",
    "                trace=int(K * (1 - noise_amount)),\n",
    "                valid_noise_matrix=False,\n",
    "                frac_zero_noise_rates=frac_zero_noise_rates,\n",
    "                seed=0,\n",
    "            )\n",
    "\n",
    "            # noise matrix is valid if diagonal maximizes row and column\n",
    "            valid = all((nm.argmax(axis=0) == range(K)) & (nm.argmax(axis=1) == range(K)))\n",
    "            print('valid:', valid)\n",
    "\n",
    "            # Create noisy labels\n",
    "            np.random.seed(seed=0)\n",
    "            s = noise_generation.generate_noisy_labels(y, nm)\n",
    "            \n",
    "            # Check accuracy of s and y\n",
    "            print('Accuracy of s and y:', sum(s==y)/len(s))\n",
    "\n",
    "            # Create map of filenames to noisy labels\n",
    "            d = dict(zip([i for i,j in train_dataset.imgs], [int(i) for i in s]))\n",
    "\n",
    "            # Store dictionary as json\n",
    "            wfn_base = '{}_noisy_labels__frac_zero_noise_rates__{}__noise_amount__{}'.format(\n",
    "                cifar_dataset,\n",
    "                \"0.0\" if frac_zero_noise_rates  < 1e-4 else round(frac_zero_noise_rates, 1),\n",
    "                \"0.0\" if noise_amount < 1e-4 else round(noise_amount, 1),\n",
    "            )\n",
    "            wfn = data_path + \"noisy_labels/\" + wfn_base\n",
    "            print(wfn)\n",
    "\n",
    "            # Store the dictionary        \n",
    "            with open(wfn + \".json\", 'w') as wf:\n",
    "                wf.write(json.dumps(d))\n",
    "\n",
    "            # Store the noise matrix as well\n",
    "            wfn_base = \"{}_noise_matrix\".format(cifar_dataset) + \"__\" + \"__\".join(wfn_base.split(\"__\")[1:])\n",
    "            wfn = data_path + \"noisy_labels/\" + wfn_base\n",
    "            print(wfn)\n",
    "            with open(wfn + \".pickle\", 'wb') as wf:\n",
    "                pickle.dump(nm, wf, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# View the noise matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Noise amount: 0.2 | Sparsity: 0.0\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.53\t0.01\t0.01\t0.0\t0.0\t0.04\t0.0\t0.01\t0.01\t0.03\n",
      "s=1 |\t0.07\t0.84\t0.03\t0.0\t0.0\t0.01\t0.03\t0.0\t0.02\t0.02\n",
      "s=2 |\t0.06\t0.02\t0.62\t0.0\t0.02\t0.04\t0.01\t0.01\t0.01\t0.01\n",
      "s=3 |\t0.06\t0.01\t0.03\t0.97\t0.01\t0.0\t0.0\t0.01\t0.0\t0.01\n",
      "s=4 |\t0.0\t0.0\t0.09\t0.0\t0.93\t0.05\t0.01\t0.01\t0.02\t0.01\n",
      "s=5 |\t0.08\t0.0\t0.04\t0.0\t0.0\t0.7\t0.01\t0.01\t0.01\t0.0\n",
      "s=6 |\t0.02\t0.01\t0.08\t0.0\t0.01\t0.11\t0.92\t0.01\t0.01\t0.01\n",
      "s=7 |\t0.0\t0.05\t0.08\t0.0\t0.0\t0.01\t0.01\t0.92\t0.0\t0.04\n",
      "s=8 |\t0.16\t0.01\t0.02\t0.0\t0.0\t0.02\t0.01\t0.0\t0.9\t0.2\n",
      "s=9 |\t0.0\t0.05\t0.0\t0.01\t0.02\t0.03\t0.0\t0.01\t0.02\t0.67\n",
      "\tTrace(matrix) = 8.0\n",
      "\n",
      "Noise amount: 0.2 | Sparsity: 0.2\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.53\t0.01\t0.01\t0.0\t0.0\t0.0\t0.0\t0.0\t0.01\t0.21\n",
      "s=1 |\t0.01\t0.84\t0.01\t0.0\t0.0\t0.06\t0.04\t0.0\t0.04\t0.01\n",
      "s=2 |\t0.03\t0.0\t0.62\t0.0\t0.01\t0.04\t0.01\t0.01\t0.01\t0.02\n",
      "s=3 |\t0.07\t0.02\t0.03\t0.97\t0.0\t0.0\t0.0\t0.01\t0.0\t0.0\n",
      "s=4 |\t0.1\t0.02\t0.04\t0.0\t0.93\t0.05\t0.03\t0.0\t0.01\t0.02\n",
      "s=5 |\t0.01\t0.02\t0.02\t0.0\t0.0\t0.7\t0.0\t0.03\t0.01\t0.05\n",
      "s=6 |\t0.19\t0.02\t0.21\t0.01\t0.01\t0.01\t0.92\t0.02\t0.0\t0.0\n",
      "s=7 |\t0.02\t0.05\t0.05\t0.01\t0.01\t0.0\t0.0\t0.92\t0.01\t0.0\n",
      "s=8 |\t0.0\t0.02\t0.01\t0.0\t0.03\t0.13\t0.0\t0.0\t0.9\t0.03\n",
      "s=9 |\t0.05\t0.01\t0.01\t0.0\t0.0\t0.02\t0.0\t0.0\t0.0\t0.67\n",
      "\tTrace(matrix) = 8.0\n",
      "\n",
      "Noise amount: 0.2 | Sparsity: 0.4\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.53\t0.03\t0.1\t0.0\t0.05\t0.02\t0.0\t0.0\t0.0\t0.0\n",
      "s=1 |\t0.06\t0.84\t0.03\t0.0\t0.0\t0.0\t0.0\t0.03\t0.04\t0.05\n",
      "s=2 |\t0.02\t0.05\t0.62\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.05\n",
      "s=3 |\t0.0\t0.01\t0.0\t0.97\t0.0\t0.0\t0.0\t0.0\t0.02\t0.16\n",
      "s=4 |\t0.04\t0.01\t0.0\t0.0\t0.93\t0.08\t0.08\t0.01\t0.02\t0.03\n",
      "s=5 |\t0.17\t0.0\t0.0\t0.0\t0.01\t0.7\t0.0\t0.0\t0.0\t0.0\n",
      "s=6 |\t0.03\t0.03\t0.03\t0.0\t0.01\t0.0\t0.92\t0.02\t0.01\t0.0\n",
      "s=7 |\t0.04\t0.01\t0.16\t0.01\t0.0\t0.0\t0.0\t0.92\t0.01\t0.0\n",
      "s=8 |\t0.01\t0.01\t0.03\t0.02\t0.0\t0.2\t0.0\t0.01\t0.9\t0.04\n",
      "s=9 |\t0.09\t0.0\t0.03\t0.01\t0.0\t0.0\t0.0\t0.0\t0.0\t0.67\n",
      "\tTrace(matrix) = 8.0\n",
      "\n",
      "Noise amount: 0.2 | Sparsity: 0.6\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.53\t0.05\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.02\t0.0\n",
      "s=1 |\t0.31\t0.84\t0.0\t0.01\t0.0\t0.0\t0.0\t0.0\t0.01\t0.0\n",
      "s=2 |\t0.06\t0.01\t0.62\t0.0\t0.03\t0.05\t0.0\t0.0\t0.0\t0.0\n",
      "s=3 |\t0.0\t0.05\t0.0\t0.97\t0.0\t0.11\t0.0\t0.0\t0.02\t0.0\n",
      "s=4 |\t0.0\t0.01\t0.0\t0.0\t0.93\t0.0\t0.0\t0.0\t0.0\t0.0\n",
      "s=5 |\t0.0\t0.01\t0.0\t0.0\t0.01\t0.7\t0.0\t0.0\t0.0\t0.15\n",
      "s=6 |\t0.02\t0.01\t0.0\t0.0\t0.0\t0.0\t0.92\t0.0\t0.04\t0.0\n",
      "s=7 |\t0.0\t0.02\t0.0\t0.0\t0.03\t0.02\t0.0\t0.92\t0.01\t0.14\n",
      "s=8 |\t0.08\t0.0\t0.38\t0.0\t0.0\t0.0\t0.08\t0.02\t0.9\t0.05\n",
      "s=9 |\t0.0\t0.0\t0.0\t0.02\t0.0\t0.11\t0.0\t0.06\t0.0\t0.67\n",
      "\tTrace(matrix) = 8.0\n",
      "\n",
      "Noise amount: 0.4 | Sparsity: 0.0\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.4\t0.04\t0.11\t0.04\t0.03\t0.01\t0.21\t0.03\t0.0\t0.0\n",
      "s=1 |\t0.18\t0.63\t0.16\t0.04\t0.02\t0.08\t0.14\t0.05\t0.01\t0.0\n",
      "s=2 |\t0.12\t0.06\t0.46\t0.04\t0.0\t0.08\t0.05\t0.06\t0.01\t0.0\n",
      "s=3 |\t0.07\t0.03\t0.05\t0.4\t0.0\t0.05\t0.09\t0.01\t0.01\t0.0\n",
      "s=4 |\t0.0\t0.04\t0.04\t0.04\t0.71\t0.01\t0.09\t0.01\t0.01\t0.0\n",
      "s=5 |\t0.08\t0.01\t0.0\t0.03\t0.08\t0.52\t0.02\t0.08\t0.01\t0.01\n",
      "s=6 |\t0.0\t0.12\t0.11\t0.04\t0.01\t0.03\t0.29\t0.05\t0.01\t0.0\n",
      "s=7 |\t0.0\t0.03\t0.03\t0.02\t0.08\t0.04\t0.02\t0.68\t0.0\t0.0\n",
      "s=8 |\t0.1\t0.01\t0.02\t0.22\t0.04\t0.11\t0.06\t0.02\t0.93\t0.01\n",
      "s=9 |\t0.04\t0.02\t0.01\t0.13\t0.03\t0.07\t0.03\t0.01\t0.02\t0.98\n",
      "\tTrace(matrix) = 6.0\n",
      "\n",
      "Noise amount: 0.4 | Sparsity: 0.2\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.4\t0.05\t0.0\t0.03\t0.01\t0.0\t0.0\t0.0\t0.0\t0.0\n",
      "s=1 |\t0.23\t0.63\t0.2\t0.04\t0.0\t0.0\t0.3\t0.04\t0.04\t0.0\n",
      "s=2 |\t0.04\t0.0\t0.46\t0.06\t0.01\t0.01\t0.0\t0.05\t0.01\t0.01\n",
      "s=3 |\t0.01\t0.04\t0.06\t0.4\t0.15\t0.15\t0.0\t0.13\t0.01\t0.0\n",
      "s=4 |\t0.0\t0.02\t0.03\t0.0\t0.71\t0.02\t0.1\t0.0\t0.01\t0.0\n",
      "s=5 |\t0.08\t0.2\t0.01\t0.03\t0.0\t0.52\t0.03\t0.08\t0.0\t0.01\n",
      "s=6 |\t0.01\t0.01\t0.02\t0.2\t0.01\t0.11\t0.29\t0.02\t0.0\t0.0\n",
      "s=7 |\t0.05\t0.03\t0.06\t0.07\t0.04\t0.04\t0.12\t0.68\t0.0\t0.0\n",
      "s=8 |\t0.06\t0.0\t0.07\t0.07\t0.0\t0.15\t0.16\t0.0\t0.93\t0.0\n",
      "s=9 |\t0.12\t0.0\t0.09\t0.1\t0.05\t0.0\t0.0\t0.0\t0.0\t0.98\n",
      "\tTrace(matrix) = 6.0\n",
      "\n",
      "Noise amount: 0.4 | Sparsity: 0.4\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.4\t0.18\t0.02\t0.04\t0.0\t0.01\t0.0\t0.0\t0.0\t0.0\n",
      "s=1 |\t0.25\t0.63\t0.05\t0.06\t0.12\t0.0\t0.26\t0.15\t0.0\t0.0\n",
      "s=2 |\t0.05\t0.0\t0.46\t0.03\t0.0\t0.0\t0.0\t0.06\t0.01\t0.0\n",
      "s=3 |\t0.01\t0.0\t0.04\t0.4\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n",
      "s=4 |\t0.0\t0.12\t0.14\t0.1\t0.71\t0.19\t0.26\t0.1\t0.04\t0.0\n",
      "s=5 |\t0.09\t0.0\t0.0\t0.02\t0.02\t0.52\t0.15\t0.02\t0.0\t0.0\n",
      "s=6 |\t0.01\t0.07\t0.13\t0.06\t0.12\t0.09\t0.29\t0.0\t0.0\t0.0\n",
      "s=7 |\t0.0\t0.0\t0.13\t0.03\t0.0\t0.13\t0.0\t0.68\t0.0\t0.0\n",
      "s=8 |\t0.06\t0.0\t0.03\t0.07\t0.0\t0.06\t0.05\t0.0\t0.93\t0.01\n",
      "s=9 |\t0.13\t0.0\t0.01\t0.19\t0.03\t0.0\t0.0\t0.0\t0.02\t0.98\n",
      "\tTrace(matrix) = 6.0\n",
      "\n",
      "Noise amount: 0.4 | Sparsity: 0.6\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.4\t0.05\t0.0\t0.04\t0.0\t0.0\t0.05\t0.0\t0.0\t0.0\n",
      "s=1 |\t0.32\t0.63\t0.0\t0.04\t0.27\t0.04\t0.0\t0.0\t0.05\t0.01\n",
      "s=2 |\t0.06\t0.0\t0.46\t0.04\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n",
      "s=3 |\t0.01\t0.04\t0.0\t0.4\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n",
      "s=4 |\t0.0\t0.02\t0.0\t0.04\t0.71\t0.0\t0.0\t0.0\t0.0\t0.0\n",
      "s=5 |\t0.11\t0.2\t0.0\t0.03\t0.0\t0.52\t0.39\t0.0\t0.03\t0.0\n",
      "s=6 |\t0.02\t0.01\t0.0\t0.04\t0.02\t0.0\t0.29\t0.0\t0.0\t0.0\n",
      "s=7 |\t0.0\t0.03\t0.0\t0.02\t0.0\t0.0\t0.0\t0.68\t0.0\t0.01\n",
      "s=8 |\t0.08\t0.0\t0.38\t0.22\t0.0\t0.0\t0.28\t0.0\t0.93\t0.0\n",
      "s=9 |\t0.0\t0.0\t0.16\t0.13\t0.0\t0.44\t0.0\t0.32\t0.0\t0.98\n",
      "\tTrace(matrix) = 6.0\n",
      "\n",
      "Noise amount: 0.7 | Sparsity: 0.0\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.2\t0.11\t0.16\t0.06\t0.1\t0.01\t0.25\t0.05\t0.01\t0.03\n",
      "s=1 |\t0.14\t0.32\t0.23\t0.06\t0.06\t0.13\t0.16\t0.07\t0.02\t0.05\n",
      "s=2 |\t0.01\t0.06\t0.23\t0.05\t0.01\t0.12\t0.06\t0.08\t0.02\t0.02\n",
      "s=3 |\t0.07\t0.1\t0.07\t0.2\t0.01\t0.08\t0.11\t0.02\t0.03\t0.01\n",
      "s=4 |\t0.07\t0.12\t0.06\t0.05\t0.14\t0.01\t0.11\t0.01\t0.02\t0.03\n",
      "s=5 |\t0.0\t0.07\t0.01\t0.04\t0.23\t0.26\t0.02\t0.12\t0.01\t0.36\n",
      "s=6 |\t0.13\t0.01\t0.16\t0.06\t0.04\t0.05\t0.14\t0.06\t0.01\t0.07\n",
      "s=7 |\t0.01\t0.13\t0.04\t0.03\t0.23\t0.06\t0.02\t0.56\t0.01\t0.01\n",
      "s=8 |\t0.15\t0.08\t0.03\t0.29\t0.11\t0.17\t0.07\t0.02\t0.83\t0.3\n",
      "s=9 |\t0.23\t0.02\t0.01\t0.17\t0.08\t0.11\t0.04\t0.01\t0.04\t0.12\n",
      "\tTrace(matrix) = 3.0\n",
      "\n",
      "Noise amount: 0.7 | Sparsity: 0.2\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.2\t0.09\t0.04\t0.0\t0.03\t0.16\t0.02\t0.0\t0.0\t0.18\n",
      "s=1 |\t0.0\t0.32\t0.1\t0.0\t0.05\t0.0\t0.08\t0.13\t0.08\t0.18\n",
      "s=2 |\t0.01\t0.04\t0.23\t0.09\t0.3\t0.02\t0.03\t0.04\t0.03\t0.0\n",
      "s=3 |\t0.0\t0.07\t0.0\t0.2\t0.01\t0.07\t0.17\t0.03\t0.01\t0.0\n",
      "s=4 |\t0.15\t0.04\t0.02\t0.0\t0.14\t0.03\t0.12\t0.12\t0.02\t0.15\n",
      "s=5 |\t0.35\t0.35\t0.0\t0.25\t0.07\t0.26\t0.05\t0.04\t0.0\t0.16\n",
      "s=6 |\t0.14\t0.02\t0.09\t0.0\t0.0\t0.2\t0.14\t0.09\t0.01\t0.15\n",
      "s=7 |\t0.06\t0.06\t0.14\t0.09\t0.3\t0.06\t0.2\t0.56\t0.0\t0.06\n",
      "s=8 |\t0.0\t0.02\t0.32\t0.22\t0.0\t0.01\t0.07\t0.0\t0.83\t0.0\n",
      "s=9 |\t0.09\t0.0\t0.04\t0.15\t0.11\t0.2\t0.12\t0.0\t0.02\t0.12\n",
      "\tTrace(matrix) = 3.0\n",
      "\n",
      "Noise amount: 0.7 | Sparsity: 0.4\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.2\t0.04\t0.0\t0.06\t0.0\t0.01\t0.0\t0.0\t0.02\t0.0\n",
      "s=1 |\t0.0\t0.32\t0.01\t0.31\t0.07\t0.06\t0.55\t0.25\t0.01\t0.0\n",
      "s=2 |\t0.0\t0.14\t0.23\t0.03\t0.07\t0.06\t0.0\t0.0\t0.03\t0.71\n",
      "s=3 |\t0.13\t0.05\t0.2\t0.2\t0.24\t0.09\t0.0\t0.0\t0.0\t0.17\n",
      "s=4 |\t0.33\t0.03\t0.0\t0.05\t0.14\t0.02\t0.0\t0.0\t0.0\t0.0\n",
      "s=5 |\t0.31\t0.0\t0.21\t0.04\t0.14\t0.26\t0.31\t0.0\t0.0\t0.0\n",
      "s=6 |\t0.03\t0.09\t0.1\t0.19\t0.2\t0.19\t0.14\t0.0\t0.11\t0.0\n",
      "s=7 |\t0.0\t0.29\t0.0\t0.06\t0.0\t0.14\t0.0\t0.56\t0.0\t0.0\n",
      "s=8 |\t0.0\t0.05\t0.04\t0.0\t0.05\t0.12\t0.0\t0.14\t0.83\t0.0\n",
      "s=9 |\t0.0\t0.0\t0.2\t0.06\t0.08\t0.05\t0.0\t0.05\t0.0\t0.12\n",
      "\tTrace(matrix) = 3.0\n",
      "\n",
      "Noise amount: 0.7 | Sparsity: 0.6\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(s|y) of shape (10, 10)\n",
      " p(s|y)\ty=0\ty=1\ty=2\ty=3\ty=4\ty=5\ty=6\ty=7\ty=8\ty=9\n",
      "\t---\t---\t---\t---\t---\t---\t---\t---\t---\t---\n",
      "s=0 |\t0.2\t0.0\t0.1\t0.06\t0.0\t0.0\t0.0\t0.01\t0.0\t0.07\n",
      "s=1 |\t0.0\t0.32\t0.01\t0.56\t0.0\t0.13\t0.0\t0.0\t0.13\t0.29\n",
      "s=2 |\t0.0\t0.05\t0.23\t0.07\t0.39\t0.0\t0.0\t0.0\t0.0\t0.0\n",
      "s=3 |\t0.0\t0.04\t0.0\t0.2\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n",
      "s=4 |\t0.0\t0.06\t0.0\t0.0\t0.14\t0.13\t0.0\t0.0\t0.0\t0.06\n",
      "s=5 |\t0.74\t0.0\t0.36\t0.0\t0.47\t0.26\t0.0\t0.08\t0.04\t0.17\n",
      "s=6 |\t0.06\t0.0\t0.24\t0.11\t0.0\t0.0\t0.14\t0.11\t0.0\t0.0\n",
      "s=7 |\t0.0\t0.36\t0.0\t0.0\t0.0\t0.12\t0.0\t0.56\t0.0\t0.18\n",
      "s=8 |\t0.0\t0.0\t0.0\t0.0\t0.0\t0.36\t0.0\t0.24\t0.83\t0.11\n",
      "s=9 |\t0.0\t0.17\t0.06\t0.0\t0.0\t0.0\t0.86\t0.0\t0.0\t0.12\n",
      "\tTrace(matrix) = 3.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create noisy labels for both CIFAR-10 and CIFAR-100\n",
    "# Store dictionary as json\n",
    "import numpy as np\n",
    "import pickle\n",
    "from cleanlab import util\n",
    "for cifar_dataset in [\"cifar10\"]:  #, \"cifar100\"]:\n",
    "    data_path = '/datasets/datasets/{}/{}/'.format(cifar_dataset, cifar_dataset)\n",
    "    for noise_amount in np.arange(0.2, 0.61, 0.2):\n",
    "        for frac_zero_noise_rates in np.arange(0, 0.61, 0.2):\n",
    "            # Print the noise matrix\n",
    "            rfn_base = '{}_noisy_labels__frac_zero_noise_rates__{}__noise_amount__{}'.format(\n",
    "                cifar_dataset,\n",
    "                \"0.0\" if frac_zero_noise_rates  < 1e-4 else round(frac_zero_noise_rates, 1),\n",
    "                \"0.0\" if noise_amount < 1e-4 else round(noise_amount, 1),\n",
    "            )\n",
    "            rfn = data_path + \"noisy_labels/\" + rfn_base\n",
    "            rfn_base = \"{}_noise_matrix\".format(cifar_dataset) + \"__\" + \"__\".join(rfn_base.split(\"__\")[1:])\n",
    "            rfn = data_path + \"noisy_labels/\" + rfn_base\n",
    "            with open(rfn + \".pickle\", 'rb') as rf:\n",
    "                nm = pickle.load(rf)\n",
    "            actual_noise = 0.7 if abs(noise_amount - 0.6) < 1e-3 else noise_amount\n",
    "            print('Noise amount:', round(actual_noise, 3), \"| Sparsity:\", round(frac_zero_noise_rates, 3))\n",
    "            util.print_noise_matrix(nm)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
